#include "PageLibPreprocessor.h"
#include "simhash/Simhasher.hpp"
#include <iostream>
#include <cmath>
using namespace std;
using namespace simhash;

PageLibPreprocessor::PageLibPreprocessor(Configuration *config, SplitTool &splitTool)
: _config(config), _splitTool(splitTool){

}

PageLibPreprocessor::~PageLibPreprocessor(){

}

//进行预处理
void PageLibPreprocessor::doProcess(){
    readInfoFromFile();
    cutRedundantPage();
    bulidInvertIndexMap();
    storeOnDisk();
}

//根据配置文件的信息读取网页库和位置偏移库的内容
void PageLibPreprocessor::readInfoFromFile(){
    string webPath = _config->parseConfig("WEBPAGE");
    string offsetPath = _config->parseConfig("OFFSET");
    ifstream ifsWeb(webPath);
    ifstream ifsOffset(offsetPath);
    string line, docid, pos, len;
    while(ifsOffset >> docid && ifsOffset >> pos && ifsOffset >> len){
        _offsetLib.insert({stoi(docid), {stoi(pos), stoi(len)}});
    }
    for(auto &i: _offsetLib){
        vector<char> buffer(i.second.second);
        ifsWeb.seekg(i.second.first);
        ifsWeb.read(buffer.data(), i.second.second);
        string buf(buffer.begin(), buffer.end());
        _pageLib.push_back(WebPage(buf, _config, _splitTool));
    }
    cout << "预处理读取网页库和位置偏移库成功\n";
}

//对冗余的网页进行去重
void PageLibPreprocessor::cutRedundantPage(){
    Simhasher Simhasher(_config->parseConfig("DICT_PATH"), _config->parseConfig("HMM_PATH"),
                        _config->parseConfig("IDF_PATH"), _config->parseConfig("STOP_WORD_PATH"));
    size_t topN = 3;
    uint64_t hashValue = 0;
    vector<uint64_t> hashList;
    vector<WebPage> newPageLib;
    int cnt = 0;
    for(auto &page: _pageLib){
        string s = page.getDocContent();
        Simhasher.make(s, topN, hashValue);
        bool flag = false;
        for(auto &i: hashList){
            if(Simhasher::isEqual(hashValue, i)){
                flag = true;
                cnt++;
                /* cout << "重复:" << page.getDocId() << endl; */
                break;
            }
        }
        if(!flag){
            newPageLib.push_back(page);
            hashList.push_back(hashValue);
        }
    }
    _pageLib = newPageLib;
    cout << "去重后网页库中的网页数: " << _pageLib.size() << "\n";
    //重新生成位置偏移库
    _offsetLib.clear();
    size_t posNow = 0;
    for(auto &i: _pageLib){
        int docid = i.getDocId();
        int pos = posNow;
        int len = i.getDoc().size();
        posNow += len;
        _offsetLib[docid] = {pos, len};
    }
    cout << "网页库去重成功\n";
}

//创建倒排索引表
void PageLibPreprocessor::bulidInvertIndexMap(){
    double IDF, w;
    map<string, int> dfMap; //每个单词的出现次数
    map<string, int> idfMap; //每个单词的IDF系数
    for(auto &page: _pageLib){
        map<string, int> wordsMap = page.getWordsMap(); //每个文档所有单词的词频表
        for(auto &word: wordsMap){
            dfMap[word.first]++;
        }
    }
    for(auto &i: dfMap){
        int N = _pageLib.size();
        IDF = log2(N/(i.second + 1));
        idfMap[i.first] = IDF;
    }

    //计算每个单词在每篇文档中的权重值
    for(auto &page: _pageLib){
        map<string, int> wordsMap = page.getWordsMap();
        map<string, double> wMap; //每个单词的权重
        for(auto &word: wordsMap){
            w = word.second * idfMap[word.first];
            wMap[word.first] = w;
        }
        //权重归一化处理
        double total = 0;
        for(auto &i: wMap){
            total += pow(i.second, 2);
        }
        double sqrtTotal = sqrt(total);
        for(auto &i: wMap){
            w = i.second / sqrtTotal;
            _invertIndexLib[i.first].push_back({page.getDocId(), w});
        }
    }
    cout << "创建倒排索引表成功\n";
}

//将经过预处理之后的网页库、位置偏移库和倒排索引表写回到磁盘上
void PageLibPreprocessor::storeOnDisk(){
    ofstream ofs1(_config->parseConfig("NEW_WEBPAGE"));
    ofstream ofs2(_config->parseConfig("NEW_OFFSET"));
    ofstream ofs3(_config->parseConfig("INVERT_INDEX"));
    //保存网页库
    for(auto &i: _pageLib){
        ofs1 << i.getDoc();    
    }
    //保存位置偏移库
    for(auto &i: _offsetLib){
        ofs2 << i.first << " " << i.second.first << " " << i.second.second << "\n";
    }
    //保存倒排索引库
    for(auto &i: _invertIndexLib){
        ofs3 << i.first << " ";
        for(auto &j: i.second){
            ofs3 << j.first << " " << j.second << " ";
        }
        ofs3 << "\n";
    }
    cout << "预处理网页库、位置偏移库和倒排索引表成功\n";
}
